# Copyright (c) 2020 Presto Labs Pte. Ltd.
# Author: leon, gekim

import json
import toml
import importlib
import sys
import itertools
import re

import requests
from absl import app
import datetime as dt
import os

from coin2.support.product_info.run import transform
from data.coin2.product_info.gen_compat import merge
from coin.exchange.binance.kr_rest.product import BinanceProduct
from coin2.support.product_info.ico import add_ico_symbol

normal_enable_quote_list = ["USDT", "USD", "KRW", "EUR", "JPY", "GBP", "BUSD", "USDC", "TRY"]

def gen_old_compat():
  merge(['data/coin2/product_info/Futures.Huobi.v1.json',
         'data/coin2/product_info/Futures.Huobi.v1-swap.json',
         'data/coin2/product_info/Spot.Huobi.v1.json',
         'data/coin2/product_info/Futures.Huobi.v1-linear-swap.json'],
        'data/coin2/product_info/old_compat/huobi.json')
  merge(['data/coin2/product_info/Futures.Binance.v1.json',
         'data/coin2/product_info/Spot.Binance.v1.json',
         'data/coin2/product_info/Futures.Binance.v1-delivery.json'],
        'data/coin2/product_info/old_compat/binance.json')
  merge(['data/coin2/product_info/Futures.Okex.v5.json',
         'data/coin2/product_info/Futures.Okex.v5-swap.json'],
        'data/coin2/product_info/old_compat/okex_v5_futures.json')
  merge(['data/coin2/product_info/Futures.Gateio.v4.json',
         'data/coin2/product_info/Futures.Gateio.v4-delivery.json'],
         'data/coin2/product_info/old_compat/gateio_futures.json'),
  merge(['data/coin2/product_info/Futures.Bybit.v2.json',
        'data/coin2/product_info/Futures.Bybit.v2-linear.json'],
        'data/coin2/product_info/old_compat/bybit_futures.json')


def filter_symbol_list_to_update(mea, symbol_list_to_update):
  binance_enable_quote_list = ["USDT", "BTC", "TUSD", "USDC",
                               "BUSD", "PAX", "EUR", "GBP", "ETH", "BNB", "TRY", "RUB", "NGN"]

  def is_quote_enable(symbol, enable_quote_list):
    for quote in enable_quote_list:
      if "-" + quote in symbol:
        return True
    return False

  if mea in ('Spot.Binance.v1'):
    return [symbol for symbol in symbol_list_to_update if is_quote_enable(symbol, binance_enable_quote_list)]
  elif mea in ('Spot.Okex.v3', 'Spot.Bithumb.v2', 'Spot.Gdax.v1',
               'Spot.Huobi.v1', 'Spot.Upbit.v1', 'Spot.Bitstamp.v2', 'Spot.Okex.v5'):
    return symbol_list_to_update
  else:
    return [symbol for symbol in symbol_list_to_update if is_symbol_ok_update(mea,symbol) and is_quote_enable(symbol, normal_enable_quote_list)]


def fetch_latest_symbol_list(mea):
  if mea in ['Futures.Gateio.v4-btc', 'Futures.Gateio.v4']:
    mod = importlib.import_module('coin2.support.product_info.' +
                                  mea_to_mea_snake_string('Futures.Gateio.v4'))
  else:
    mod = importlib.import_module('coin2.support.product_info.' + mea_to_mea_snake_string(mea))
  pis = mod.generate()
  if mea in ['Futures.Gateio.v4-btc', 'Futures.Gateio.v4']:
    return separate_futures_Gateio_pi(mea, [pi['symbol'] for pi in pis])
  else:
    return [pi['symbol'] for pi in pis]


def find_binance_spot_top_30():
  tickers = requests.get('https://api.binance.com/api/v1/ticker/24hr').json()
  preprocessed_products = []
  for ticker in tickers:
    product = BinanceProduct.FromStrNativeProductNothrow(ticker['symbol'])
    if product is not None and product.quote.currency in ('USDT', 'USD'):
      preprocessed_products.append({
          'symbol': product.symbol,
          'volume': float(ticker['quoteVolume'])
      })
  top30 = sorted(preprocessed_products, key=lambda x: x['volume'], reverse=True)[:30]
  return [product['symbol'] for product in top30]


def find_binance_futures_top_30():
  tickers = requests.get('https://fapi.binance.com/fapi/v1/ticker/24hr').json()
  top30 = sorted(tickers, key=lambda x: float(x['quoteVolume']), reverse=True)[:30]
  return ['{symbol}.PERPETUAL'.format(symbol=product['symbol']) for product in top30]


def extract_symbol_list_from_pi(mea):
  pi_path = 'data/coin2/product_info/{mea}.json'.format(mea=mea)
  if mea in ['Futures.Gateio.v4-btc', 'Futures.Gateio.v4']:
    pi_path = 'data/coin2/product_info/Futures.Gateio.v4.json'
  with open(pi_path, 'r') as f:
    pis = json.load(f)['product_infos']
  if mea in ['Futures.Gateio.v4-btc', 'Futures.Gateio.v4']:
    return separate_futures_Gateio_pi(mea, [pi['symbol'] for pi in pis])
  else:
    return [pi['symbol'] for pi in pis]


def separate_futures_Gateio_pi(mea, product_list):
  #Futures.Gateio.v4 and .v4-btc share one pi file
  if mea == 'Futures.Gateio.v4':
    return [pi for pi in product_list if re.match('.*-USD\.', pi) is None]
  else:
    return [pi for pi in product_list if re.match('.*-USD\.', pi) is not None]


def compare_symbol_groups_pi(mea):
  pi_symbol_list = extract_symbol_list_from_pi(mea)
  symbol_groups_symbol_list = list(itertools.chain.from_iterable(
      extract_symbol_group_dict_from_toml(mea).values()))
  enable_quote_list = normal_enable_quote_list
  if mea in ["Spot.Upbit.v1","Spot.Bithumb.v2","Spot.Coinone.v2","Spot.Korbit.v1"]:
    enable_quote_list = normal_enable_quote_list.copy()
    enable_quote_list.append("BTC")
  elif mea ==  "Spot.Upbitid.v1":
    enable_quote_list = normal_enable_quote_list.copy()
    enable_quote_list.append("BTC")
    enable_quote_list.append("IDR")
  elif mea == "Spot.Mercado.v4":
    enable_quote_list = normal_enable_quote_list.copy()
    enable_quote_list.append("BRL")

  def is_quote_enable(symbol):
    if symbol in symbol_groups_symbol_list:
      return False
    for quote in enable_quote_list:
      if re.match('.*-%s\..*$' % quote,symbol) is not None or re.match('.*-%s$' % quote,symbol) is not None :
        return True
    return False

  symbol_list_to_update = [symbol for symbol in pi_symbol_list if is_symbol_ok_update(mea,symbol) and is_quote_enable(symbol)]
  if mea == 'Spot.Coingecko.v3':
    symbol_list_to_update = [symbol for symbol in pi_symbol_list if symbol in get_coingecko_symbol_universe() and is_quote_enable(symbol)]
  symbol_list_to_remove = [
    symbol for symbol in symbol_groups_symbol_list if symbol not in pi_symbol_list and is_symbol_ok_remove(symbol, mea)]  
  return symbol_list_to_update, symbol_list_to_remove


def is_symbol_ok_update(mea,symbol):
  if mea == 'Spot.Mexc.v3':
    if re.search(r'\d[S|L]-',symbol) is not None:
      return False
  elif mea == 'Spot.Gdax.v1':
    if 'USDC' in symbol:
      return False
  elif mea == 'Spot.Gateio.v4':
    if '-TRY' in symbol:
      return False
  return True

def is_symbol_ok_remove(symbol, mea):
  if 'MONTH' in symbol or 'QUARTER' in symbol or 'WEEK' in symbol:
    return False
  ico_path = "../coin_product_info/ico/%s.json" % mea
  if os.path.exists(ico_path) :
    with open(ico_path, 'r') as f:
      pis = json.load(f)
    for product_info in pis['product_infos']:
      if symbol == product_info['symbol']:
        return False
  if mea == "Futures.Prex.v1" and symbol in ["APE-USD.PERPETUAL", "GMT-USD.PERPETUAL", "IMX-USD.PERPETUAL", "RUNE-USD.PERPETUAL", "USDT-USD.PERPETUAL", "ETH2-USD.PERPETUAL"] :
    return False
  if mea == "Spot.Gateio.v4" and (symbol in ["AMPL-USDT"]):
    return False
  return True

def get_symbol_list_update(mea):
  old_symbol_list = extract_symbol_list_from_pi(mea)
  if mea in ['Spot.Mercado.v4', 'Spot.Coingecko.v3']:
    ref_symbol_list = old_symbol_list
  else:
    ref_symbol_list = fetch_latest_symbol_list(mea)
  symbol_list_to_update = [
      ref_symbol for ref_symbol in ref_symbol_list if ref_symbol not in old_symbol_list]
  symbol_list_to_update = filter_symbol_list_to_update(mea, symbol_list_to_update)
  symbol_list_to_remove = [
      ref_symbol for ref_symbol in old_symbol_list if ref_symbol not in ref_symbol_list and is_symbol_ok_remove(ref_symbol, mea)]
  symbol_group_dict = extract_symbol_group_dict_from_toml(mea)
  symbol_group_dict_to_list = list(itertools.chain.from_iterable(symbol_group_dict.values()))
  symbol_list_to_update = [
      symbol for symbol in symbol_list_to_update if symbol not in symbol_group_dict_to_list]
  symbol_list_to_update = list(set(symbol_list_to_update))
  symbol_list_to_remove = list(set(symbol_list_to_remove))
  return symbol_list_to_update, symbol_list_to_remove


def get_symbol_list_update_from_ref(mea, ref_symbol_list):
  latest_symbol_list = fetch_latest_symbol_list(mea)
  if mea == 'Spot.Binance.v1':
    # Replace quote to be USDT.
    updated_ref_symbol_list = []
    for symbol in ref_symbol_list:
      base, quote = symbol.rsplit('-', 1)
      updated_ref_symbol_list.append('{base}-{quote}'.format(base=base, quote='USDT'))
      updated_ref_symbol_list.append('{base}-{quote}'.format(base=base, quote='USD'))
  else:
    updated_ref_symbol_list = ref_symbol_list
  symbol_list_to_update = [
      ref_symbol for ref_symbol in updated_ref_symbol_list if ref_symbol in latest_symbol_list]
  symbol_list_to_update = filter_symbol_list_to_update(mea, symbol_list_to_update)
  symbol_group_dict = extract_symbol_group_dict_from_toml(mea)
  symbol_group_dict_to_list = list(itertools.chain.from_iterable(symbol_group_dict.values()))
  symbol_list_to_update = [
      symbol for symbol in symbol_list_to_update if symbol not in symbol_group_dict_to_list]
  symbol_list_to_update = list(set(symbol_list_to_update))
  return symbol_list_to_update, []


def get_symbol_group_dict_update(mea, ref_symbol_list=None):
  if ref_symbol_list is not None:
    assert mea in ('Spot.Binance.v1', 'Futures.Binance.v1'), mea
    symbol_list_to_update, symbol_list_to_remove = get_symbol_list_update_from_ref(
        mea, ref_symbol_list)
  else:
    symbol_list_to_update, symbol_list_to_remove = get_symbol_list_update(mea)

  symbol_group_dict = extract_symbol_group_dict_from_toml(mea)

  symbol_list_update_symbol_groups_pi, symbol_list_remove_symbol_groups_pi = compare_symbol_groups_pi(
      mea)
  symbol_list_to_update += symbol_list_update_symbol_groups_pi
  symbol_list_to_update = list(set(symbol_list_to_update))
  symbol_list_to_remove += symbol_list_remove_symbol_groups_pi
  symbol_list_to_remove = list(set(symbol_list_to_remove))

  union_symbol_set = set(symbol_list_to_update) & set(symbol_list_to_remove)
  if len(union_symbol_set) > 0:
    symbol_list_to_update = list(set(symbol_list_to_update) - union_symbol_set)
    symbol_list_to_remove = list(set(symbol_list_to_remove) - union_symbol_set)

  for symbol in symbol_list_to_update:
    group = min(symbol_group_dict.keys(), key=(lambda k: len(symbol_group_dict[k])))
    symbol_group_dict[group].append(symbol)

  real_symbol_list_to_remove = []
  for symbol in symbol_list_to_remove:
    for group, dict_symbols in symbol_group_dict.items():
      if symbol in dict_symbols:
        dict_symbols.remove(symbol)
        real_symbol_list_to_remove.append(symbol)
        break
  print_update_info(mea, symbol_list_to_update, real_symbol_list_to_remove)

  return symbol_group_dict


def mea_to_mea_snake_string(mea):
  return mea.lower().replace('.', '_')


def extract_symbol_group_dict_from_toml(mea):
  symbol_groups_path = 'data/coin2/feed/symbol_groups.toml'
  with open(symbol_groups_path, 'r') as f:
    symbol_groups = toml.load(f)
    return symbol_groups[mea_to_mea_snake_string(mea)]


def update_symbol_groups_toml(mea, symbol_group_dict):
  symbol_groups_path = 'data/coin2/feed/symbol_groups.toml'
  with open(symbol_groups_path, 'r') as f:
    symbol_groups = toml.load(f)
    symbol_groups[mea_to_mea_snake_string(mea)] = symbol_group_dict
    if mea == 'Spot.Mexc.v3':
      symbol_groups['spot_mexc_v2'] = symbol_group_dict

  with open(symbol_groups_path, 'w') as f:
    toml.dump(symbol_groups, f)


def format_symbol_groups_toml():
  symbol_groups_path = 'data/coin2/feed/symbol_groups.toml'
  with open(symbol_groups_path, 'r') as f:
    symbol_groups = toml.load(f)

  with open(symbol_groups_path, 'w') as f:
    toml.dump(symbol_groups, f)


def update_product_info(mea):
  if mea == 'Futures.Gateio.v4-btc':
    mea = 'Futures.Gateio.v4'
  if mea in ['Spot.Mercado.v4']:
    return
  mod = importlib.import_module('coin2.support.product_info.' + mea_to_mea_snake_string(mea))
  pis = mod.generate()

  with open(f'data/coin2/product_info/{mea}.json', 'w') as f:
    json.dump(transform(mea, pis), f, indent=2)
  add_ico_symbol(mea,'../coin_product_info')
  if mea in ('Spot.Binance.v1'):
    add_binance_spot_loka()


def get_coingecko_symbol_universe():
  product = [x.split('-')[0] + '-USD' for x in extract_symbol_list_from_pi('Futures.Binance.v1')] + \
  [x.split('-')[0] + '-USD' for x in extract_symbol_list_from_pi('Futures.Okex.v5-swap')] + \
  [x.split('-')[0] + '-USD' for x in extract_symbol_list_from_pi('Futures.Bybit.v3-linear')]
  return set(product)


def add_binance_spot_loka():
  pisf = "data/coin2/product_info/Spot.Binance.v1.json"
  loka = "cc/experimental/wliang/product_info/Spot.Binance.v1-LOKA.json"
  with open(pisf) as fd:
    doc = json.load(fd)
    for info in doc["product_infos"]:
      if info["base"] == "LOKA":
        return
  mea = None
  fetched_time = None
  lists = []
  with open(pisf) as fd:
    doc = json.load(fd)
    mea = doc['mea']
    fetched_time = doc['fetched_time']
    lists.extend(doc['product_infos'])
  with open(loka) as fd:
    doc = json.load(fd)
    lists.extend(doc['product_infos'])

  doc = {
      "mea": mea,
      "fetched_time": fetched_time,
      "product_infos": lists,
  }
  with open(pisf, 'w') as fd:
    json.dump(doc, fd, indent=2)


def print_update_info_title():
  print(dt.datetime.now().strftime("%Y.%m.%d"), " Daily Update Report")
  print("The vanilla on")
  print("  feed-05.ap-northeast-1.aws")
  print("and the fastfeed_converter have been deployed.")
  print("")


def print_update_info(mea, symbol_list_to_update, symbol_list_to_remove):
  if len(symbol_list_to_update) <= 0 and len(symbol_list_to_remove) <= 0:
    return
  if mea == "Spot.Mexc.v3":
    mea = "Spot.Mexc.v2/v3"
  print("-------------------", mea, "-----------------")
  if len(symbol_list_to_update) > 0:
    print("Add symbols:", ",".join(symbol_list_to_update))
    print("")
  if len(symbol_list_to_remove) > 0:
    print("REMOVE symbols:", ",".join(symbol_list_to_remove))
    print("")


def main(argv):
  meas = [
      'Spot.Bithumb.v2',
      'Spot.Upbit.v1',
      'Spot.Coinone.v2',
      'Spot.Gdax.v1',
      'Spot.Bitstamp.v2',
      'Spot.Binance.v1',
      # 'Spot.Ftx.v1',
      # 'Spot.Okex.v3',
      'Spot.Huobi.v1',
      'Futures.Binance.v1',
      'Futures.Binance.v1-delivery',
      'Futures.Huobi.v1',
      'Futures.Huobi.v1-swap',
      'Futures.Huobi.v1-linear-swap',
      # 'Futures.Okex.v3',
      # 'Futures.Okex.v3-swap',
      # 'Futures.Ftx.v1',
      'Futures.Bybit.v2',
      'Futures.Bybit.v2-linear',
      'Futures.Bybit.v3',
      'Futures.Bybit.v3-linear',
      'Futures.Bybit.v3-linear-usdc',
      'Spot.Korbit.v1',
      'Futures.Prex.v1',
      'Spot.Bybit.v1',
      'Spot.Bybit.v3',
      'Spot.Phemex.v1',
      'Spot.Okex.v5',
      'Futures.Okex.v5',
      'Futures.Okex.v5-swap',
      'Spot.Kucoin.v1',
      'Futures.Kucoin.v1',
      'Futures.Bitmex.v1',
      'Spot.Bitbank.v1',
      'Spot.Gateio.v4',
      'Futures.Gateio.v4',
      'Futures.Gateio.v4-btc',
      'Spot.Upbitid.v1',
      'Spot.Upbitsg.v1',
      'Spot.Mexc.v3',
      'Futures.Mexc.v1',
      'Spot.Binanceus.v1',
      'Spot.Crypto.v2',
      'Futures.Dydx.v3',
      'Spot.Btcturk.v1',
      'Futures.Phemex.v1',
      'Futures.Crypto.v1',
      'Spot.Kraken.v1',
      # 'Spot.Bitget.v1',
      # 'Futures.Bitget.v1',
      'Futures.Apollox.v1',
      # 'Spot.Mercado.v4',
      'Spot.Coingecko.v3',
      # 'Spot.Lbank.v2',
      # 'Spot.Bitmart.v3',
  ]
  is_updated = False
  symbol_list_update_all = []
  print_update_info_title()
  update_product_info('Spot.Coingecko.v3')

  for mea in meas:
    symbol_list_update, symbol_list_remove = get_symbol_list_update(mea)
    symbol_list_update_symbol_groups_pi, symbol_list_remove_symbol_groups_pi = compare_symbol_groups_pi(
        mea)
    symbol_list_update += symbol_list_update_symbol_groups_pi
    symbol_list_update = list(set(symbol_list_update))
    symbol_list_remove += symbol_list_remove_symbol_groups_pi
    symbol_list_remove = list(set(symbol_list_remove))

    if symbol_list_update or symbol_list_remove:
      symbol_list_update_all.extend(symbol_list_update)
      is_updated = True
      symbol_group_dict = get_symbol_group_dict_update(mea)
      update_symbol_groups_toml(mea, symbol_group_dict)
      if (mea != 'Spot.Coingecko.v3'):
        update_product_info(mea)
  if is_updated:
    # Check if any updated product is available in Binance Spot.
    binance_spot_mea = 'Spot.Binance.v1'
    symbol_group_dict = get_symbol_group_dict_update(
        binance_spot_mea, ref_symbol_list=symbol_list_update_all)
    update_symbol_groups_toml(binance_spot_mea, symbol_group_dict)

  # Update for Binance volume top 30.
  mea = 'Spot.Binance.v1'
  symbol_list_update, _ = get_symbol_list_update_from_ref(mea, find_binance_spot_top_30())
  if symbol_list_update:
    is_updated = True
    symbol_group_dict = get_symbol_group_dict_update(
        mea, ref_symbol_list=symbol_list_update)
    update_symbol_groups_toml(mea, symbol_group_dict)

  mea = 'Futures.Binance.v1'
  symbol_list_update, _ = get_symbol_list_update_from_ref(mea, find_binance_futures_top_30())
  if symbol_list_update:
    is_updated = True
    symbol_group_dict = get_symbol_group_dict_update(
        mea, ref_symbol_list=symbol_list_update)
    update_symbol_groups_toml(mea, symbol_group_dict)

  if is_updated:
    gen_old_compat()
    sys.stdout.write('updated')
  else:
    sys.stdout.write('noupdate')


if __name__ == '__main__':
  app.run(main)
